<?php
// +----------------------------------------------------------------------
// | 杭州十禾信息科技有限公司
// +----------------------------------------------------------------------
// | Copyright (c) 2013-2015 http://www.shelpe.com All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: LeePuman <ysyzlsm@163.com>
// +----------------------------------------------------------------------

/**
 * 直通车报表数据API
 */

class RptKeywordModel extends RptBaseModel {

	// protected $tableName = 'rpt_keyword';
	protected $tableName = null;
	/** @var object */
	private $me = null;
	/** @var object */
	private $subwayToken = null;

  	private $cache_keyword = null;

	/**
	 * 初始化。主要检查session['me']。
	 *
	 */
	public function _initialize()
	{
		parent :: _initialize();
		$this->me = session('me');
		$this->tableName = "rpt_keyword{$this->me['user_id']}";
    	$this->cache_keyword = Cache::getInstance('File');
	}
	

	/**
	 * @deprecated 使用  self::getSummaryStats() 或者  self::getSeriesStats().
	 * 
	 * 根据user_id, campaign_id, 或者adgroup_id分页查询关键词的统计信息. 
	 * <strong>注意: </strong>因为这里是报表数据, 因此获取到的数据可能包含有已经被删除掉的关键词.
	 *  
	 * @param string $type 下列3个中的一个: 'adgroup_id', 'campaign_id', 'taobao_user_id'.
	 * @param int $vid 要查询的type的ID. 
	 * @param string('Y-m-d') $from 统计起始日期(包含). 这个字段没有进行安全处理, 调用的时候必须确保合法!!!
	 * @param string('Y-m-d') $to 统计终止日期(包含). 这个字段没有进行安全处理, 调用的时候必须确保合法!!!
	 * @param string $order 要排序的字段. 这个字段没有进行安全处理, 调用的时候必须确保合法!!!
	 * @param string $desc 'desc' 或者 'asc'. 大小写均可. 这个字段没有进行安全处理, 调用的时候必须确保合法!!!
	 * @param int $page 页数, 从1开始.
	 * @param int $pageSize 每页显示的记录条数
	 * @param array $filters 过滤字段 TODO 目前的参数中没有处理该参数!!!
	 * @param bool(default=false) $idAsKey [可选]是否将keyword_id做为返回数组的(字符串)主键 . 默认为false.
	 * @return array2D 返回2维数组 : array( keyword_id => array(...) )
	 * @throws ThinkException 参数错误或者查询出错时抛出异常
	 */
	public function getKeywordStat($type, $vid, $from , $to , $order = 'impressions' , $desc = 'desc', 
			$page = 1, $pageSize = 200, $filters = array(), $idAsKey = false)
	{
		
		//return $this->getSummaryStats($this->me['user_id'], $type, $vid, 'all', $from , $to);
		
		if(!in_array($type, array('adgroup_id', 'campaign_id', 'taobao_user_id'))) {
			throw_exception('type字段无效');
		}
		$vid = intval($vid); //确保为整型
		if(!$vid) {
			throw_exception('ID值无效');
		}
		if(!$from || !$to || !$order || !in_array(strtolower($desc), array('desc', 'asc')) 
				|| $page <= 0 || $pageSize < 0) {
			throw_exception('from, to, roder, desc, page, pageSize这几个参数中的一个或几个有误');
		}
		
		$sqlLimit = $pageSize == 0 ? '' : "LIMIT" .(($page - 1) * $pageSize) . ",$pageSize";
		$sql = "
SELECT keywordstr as word,keyword_id, adgroup_id,campaign_id, avg(ctr) as ctr , avg(cpc)/100 as cpc, avg(cpm)/100 as cpm,
	sum(impressions) as impressions,sum(click) as click,
	sum(cost)/100 as cost, sum(avgpos*impressions) as avgpos, sum(direct_pay + indirect_pay)/100 as sales, sum(direct_pay)/100 as direct_pay,
	sum(indirect_pay)/100 as indirect_pay, sum(direct_pay_count) as direct_pay_count,
	sum(indirect_pay_count) as indirect_pay_count,sum(fav_shop_count) as fav_shop_count,
	sum(fav_item_count) as fav_item_count 
FROM `rpt_keyword{$this->me['user_id']}` where $type = $vid and `date` >= '$from' and `date` <= '$to' AND source IN (1,2) 
GROUP BY keyword_id ORDER BY $order $desc 
 " . $sqlLimit;
		$res = $this->query($sql);//todo: check if return an array or object
		
		if($res === false) {
			throw_exception('SQL查询出错: ' . $sql);
		} elseif(empty($res)) {
			return array();
		} else {
			foreach($res as &$k){
				// mysql统计出来的roi不对, 这里特殊处理.
				$k['roi'] = ($k['sales'] > 0 && $k['cost'] > 0) ? round($k['sales'] / $k['cost'], 2) : 0;
				// mysql统计出来的cpc不对, 这里特殊处理.
				$k['cpc'] = ($k['click'] > 0 && $k['cost'] > 0) ? round($k['cost'] / $k['click'], 2) : 0;
				// mysql统计出来的ctr不对, 这里特殊处理.
				$k['ctr'] = ($k['impressions'] > 0 && $k['click'] > 0) ? round($k['click'] / $k['impressions'], 4)*100 : 0;
				// mysql统计出来的cpm不对, 这里特殊处理.
				$k['cpm'] = ($k['impressions'] > 0 && $k['cost'] > 0) ? round($k['cost'] / $k['impressions']*1000, 2) : 0;
				// mysql统计出来的avgpos不对, 这里特殊处理.
				$k['avgpos'] = $k['impressions'] > 0 ? round($k['avgpos'] / $k['impressions'], 2) : 0;
			}
			
			if(!$idAsKey) {
				return $res;
			} else {
				$newRes = array();
				foreach($res as $rec) {
					$newRes[(string) $rec['keyword_id']] = $rec;
				}
				return $newRes;
			}
		}
	}
	
	/**
	 * 通过过滤器获取关键词信息
	 * @param string $type ，支持'campaign_id' ,'adgroup_id' , 'taobao_user_id'
	 * @param int $vid , 对应的第一个参数类型数字ID 
	 * @param mixed $filters ，过滤器 , 目前支持如下几个常用过滤方式，可以输入多个, 例如'pv,click,sales'
	 * 		'pv' => 有浏览的关键词
	 * 		'click' => 有点击的关键词
	 * 		'cost' => 有点击花费的关键词
	 * 		'sales' => '有成交的关键词'
	 * 		'fav' => '有收藏的关键词'
	 * 
	 */
	public function getKeywordsByFilters($type , $vid , $filters , $from , $to)
	{
		if(!in_array($type, array('adgroup_id', 'campaign_id', 'taobao_user_id'))) {
			throw_exception('type字段无效');
		}
		$vid = intval($vid); //确保为整型
		if(!$vid) {
			throw_exception('ID值无效');
		}		
		if(!$from || !$to ) {
			throw_exception('需要输入起始时间和结束时间');
		}
		
		$filters_mapper = array(
				'pv' => ' impressions > 0',
				'click' => ' click > 0 ',
				'cost' => ' cost > 0 ',
				'sales' => ' (direct_pay > 0 OR indirect_pay > 0) ',
				'fav' => ' (fav_shop_count > 0 OR fav_item_count > 0) '
		);
		
		$filter_sql = '';
		if($filters) {
			$tmp_sql = array();
			$filters = is_array($filters) ? $filters : explode(',' , $filters);
			foreach( $filters as $filter){
				if( isset($filters_mapper[$filter]) ){
					$tmp_sql[] = $filters_mapper[$filter];
				}
			}
			$filter_sql = implode(' AND ' , $tmp_sql);
		}
		
		$whrIds = is_array($vid) ? "$type IN ( " . join(',', $vid) . ") " :  "$type = '$vid'";
		
		$sql = "
			SELECT keyword_id,keywordstr,
			SUM(impressions) AS impressions,SUM(click) AS click,SUM(cost)/100 AS cost,
			SUM(direct_pay + indirect_pay)/100 AS sales,
			SUM(direct_pay)/100 AS direct_pay,SUM(indirect_pay)/100 AS indirect_pay,
			SUM(direct_pay_count) AS direct_pay_count,
			SUM(indirect_pay_count) AS indirect_pay_count,
			SUM(fav_shop_count) AS fav_shop_count,
			SUM(fav_item_count) AS fav_item_count,
			100*SUM(direct_pay_count + indirect_pay_count)/SUM(click) AS cvr,
			100*SUM(click)/SUM(impressions) AS ctr, 
			SUM(cost)/100/SUM(click) AS cpc, 
			10*SUM(cost)/SUM(impressions) AS cpm,
			AVG(avgpos) AS avgpos, 
			SUM(direct_pay + indirect_pay)/SUM(cost) AS roi
			FROM `rpt_keyword{$this->me['user_id']}`
			WHERE $whrIds AND `date` BETWEEN '$from' AND '$to' AND source IN (1,2)";
		if( $filter_sql ){
			$sql .= " AND " . $filter_sql;
		}
		$sql .= " GROUP BY keyword_id";
		
		$res = $this->query( $sql );
		return $res;
	}
	
  	/**
  	 * 检查给定用户给定推广计划给定source的全部关键词本次更新需要的时间段.
  	 * 
  	 * @param int $user_id
  	 * @param int $campaign_id
  	 * @param String $source '1,2' 或者 'SUMMARY'.
  	 * @return string 返回 需要更新的时间长度: DAY, 7DAY, 15DAY, 30DAY, 90DAY 或者空字符串. 
  	 * 		如果返回空字符串, 表示无需更新.
  	 */
	public static function getCampKeywordTimeSlot($user_id, $campaign_id, $source) {
		$entityPrefix = 'rpt-ad-src-keyword:';
		$adgroupMdl = new AdgroupModel;
		$adgroups = $adgroupMdl->getCampaignsAdgroups($user_id, $campaign_id);
		if(!$adgroups) {
			return ''; // 这里如果能够确保DB中的adgroups是最新的话, 可以返回空字符串''.
		} else {
			$entities = array();
			foreach($adgroups as $adgroup) {
				$entities[] = $entityPrefix . $adgroup['adgroup_id'] . ':' . substr($source, 0, 3);
			}
			$lastUpdated = M('sync_status')
					->where(array(
							'user_id' => $user_id, 
							'entity' => array('IN', $entities))
					)->min('updated');
			Log::record(M('sync_status')->getLastSql(), Log::DEBUG);

			if($lastUpdated > 0){
				$timeSlot = 'DAY';
				// $lastUpdated = date('Y-m-d', strtotime($lastUpdated));
				// $lastUpdated = new DateTime($lastUpdated);

				if($lastUpdated < date('Y-m-d', time() - 31 * 86400)) {
					$timeSlot='90DAY';
				} elseif ($lastUpdated < date('Y-m-d',time() - 16 * 86400)) {
					$timeSlot='30DAY';
				} elseif ($lastUpdated < date('Y-m-d',time() - 8 * 86400)) {
					$timeSlot='15DAY';
				} elseif ($lastUpdated < date('Y-m-d',time() - 2 * 86400)) {
					$timeSlot='7DAY';
				}
				
				return $timeSlot;
			} else {
				return '7DAY';
			}
		}
	}
  	
	/**
	 * 保存给定用户的给定adgroups的最后更新时间
	 * 
	 * @param int $user_id 用户ID
	 * @param string $source '1,2' 或者 'SUMMARY'
	 * @param array(int) $adgroup_ids 要更新的adgroup的ID数组
	 * @param string $date	[可选]最后更新的日期. 格式为: Y-m-d. 默认为昨天零时. 
	 * 		TODO 在凌晨的时候更新, 可能还没有昨天的统计数据, 这是应该记录为前天零时.
	 * @return false|int 返回false如果更新失败, 否则返回插入的记录条数.
	 */
	public function saveCampKeywordTimeSlot($user_id, $source, $adgroup_ids, $date = 0) {
		$entityPrefix = 'rpt-ad-src-keyword:';
		//$date = $date ? $date : date('Y-m-d', time() - 86400);
		$data = array();
		$adgroupMdl= new AdgroupModel();
		foreach($adgroup_ids as $adgroup_id) {
			$adexist=$adgroupMdl->getAdgroupByAdgroupId($user_id,$adgroup_id);
			if($adexist){
				$record = array();
				$date = $this->getMaxDate($user_id, 'adgroup_id', $adgroup_id);
				$record['user_id'] = $user_id;
				$record['entity'] = $entityPrefix . $adgroup_id . ':' . substr($source, 0, 3);
				$record['updated'] = $date;
				$data[] = $record;
			}
		}
		return M('sync_status')->addAll($data, array(), true);
	}

	/**
	 * 同步更新某个adgroup的所有关键词报表. 非异步调用, 会直接返回结果.
	 * 
	 * @param int $user_id
	 * @param int $campaign_id
	 * @param int $adgroup_id
	 * @param bool $force //是否强制更新
	 * @return boolean
	 */
	public function syncAdgroupKeywordReport($user_id, $campaign_id , $adgroup_id , $force = false)
	{
		//强制更新时时间起始无视系统sync_status状态,直接更新最近30天的数据
		$source_type = 'SUMMARY'; // 1,2表示站内和站外; SUMMARY表示summary
		if($force){
			$time_slot = array(
				'start' => date('Y-m-d' ,  strtotime( '-30 day')),
				'end' =>  date('Y-m-d' ,  strtotime( '-1 day')) 			
			);
			$api_params = array('source' => $source_type);
		} else {
			$entity = 'rpt-ad-src-keyword:' . $adgroup_id . ':' . substr($source_type, 0, 3);
			$api_params = array('source' => $source_type);
			$time_slot = SyncModel::getSyncTimeSlot($user_id, $entity);
		}

		if($time_slot){
			//最多一次只能更新30天的数据比较合理
			if( strtotime( $time_slot['start'] ) < date2time(-30)) {
				$time_slot['start'] = date('Y-m-d' ,  strtotime( '-30 day'));
			}
			$rptMdl = new RptModel();
			$ret = $rptMdl->syncSummaryData($user_id, 'rpt_keyword', $campaign_id , $adgroup_id, 
			$time_slot['start'] , $time_slot['end'], $api_params);
			$condition = array(
				'user_id' => $user_id,
				'camapaign_id' => $campaign_id,
				'adgroup_id' => $adgroup_id,
			);
			// 更新有数据时插入成功的时间
			if($ret){
				self::saveCampKeywordTimeSlot($user_id, $source_type, array($adgroup_id));
			}
		} else {
			return false;
		}
		return true;
	}
  
	/**
	 * 将API获得的字段map到数据库表字段. 会trim掉所有字段名两头的空白字符.
	 * @param array(string)|string $fields 需要map的字段列表. 字段数组, 或者','拼接的字符串. 最好是数组.
	 */
	private function mapRptFields(&$fields)
	{
		$fieldMap = array(
				'campaignid' => 'campaign_id',
				'adgroupid' => 'adgroup_id',
				'keywordid' => 'keyword_id',
				'searchtype' => 'search_type',
				'directpay' => 'direct_pay',
				'indirectpay' => 'indirect_pay',
				'directpaycount' => 'direct_pay_count',
				'indirectpaycount' => 'indirect_pay_count',
				'favitemcount' => 'fav_item_count',
				'favshopcount' => 'fav_shop_count',
				'directcarttotal' => 'direct_cart_total',
				'indirectcarttotal' => 'indirect_cart_total',
				'carttotal' => 'cart_total'
		);
		
		if(!is_array($fields))
			$fields = explode(',', $fields);
		
		foreach ($fields as $key => $field) {
			$field = trim($field);
			$fields[$key] = isset($fieldMap[$field]) ? $fieldMap[$field] : $field;
		}
	}
	
	/**
	 * read the title line of a keyword report
	 * @param resource $handle File handle
	 * @param bool $finished Value to set: whether reaches the end of the file.
	 * @return array(string) Return the mapped titles in an array. If nothing read, empty array is returned.
	 */
	private function readTitleLine(&$handle)
	{
		if(feof($handle)) {
			return array();
		}
		
		$line = fgets($handle);
		$line = trim(preg_replace('/\s+/', ' ', $line));
		if(substr($line, 0, 1) != 'n') { // remove the first non-breaking space: 0xEF 0xBB 0xBF, ie. \ufeff. TODO find a better way!!!
			$line = substr($line, 3);
		}
		if($line) {
			$title = explode(',', $line);
			$this->mapRptFields($title);
			return $title;
		} else {
			return array();
		}
	}

	/**
	 * Read a data line in a keyword report, and set the eof flag.
	 * 
	 * @param resource $handle File handle
	 * @param bool $finished Value to set: whether reaches the end of the file
	 * @param array $title The title line in kv array.
	 * @param int $user_id
	 * @return array
	 */
	private function readDataLine(&$handle, $title, $user_id)
	{
		if(feof($handle)) {
			return array();
		}
		
		$searchTypeVal = RptBaseModel::$_SEARCH_TYPES;
		$sourceTypeVal = RptBaseModel::$_SOURCE_TYPES;
	
		$line = fgets($handle);
		$line = trim($line);
		if($line == '') {
			return array();
		}
		
		$data = explode(',', $line);
		$data = array_combine($title, $data);
		
		if($data === false) {
			throw_exception('数据错误. 数据列与title列数量不一致. 当前数据为: ' . $line);
		}
			
		$data['source'] = $sourceTypeVal[$data['source']];
		$data['search_type'] = $searchTypeVal[$data['search_type']];
		$data['taobao_user_id'] = $user_id;
		unset($data['nick']);
		
		return $data;
	}
	
	/**
	 * 读取异步api下载下来的报表数据
	 *
	 *@param resource $handle File handle
	 * @param bool $finished Value to set: whether reaches the end of the file
	 * @param array $title The title line in kv array.
	 * @param int $user_id
	 * @param array $lastLine KV array.
	 * @param array $currLine KV array.
	 * @param array(int) $adgroup_ids 如果给定的话, 会将数据中的adgroup_id保存到这个数组中.
	 * @return array2D 读取到的数据 array(array(field => value) )
	 */
	private function readOneDayData(&$handle, &$finished, $title, $user_id, &$lines, &$adgroup_ids = null)
	{
		$datalist = array();
		
		$linesNum = count($lines);
		
		while (!$finished) {
			// 读取数据
			if($lines[0]) { // not first time
				$lines[$linesNum - 1] = $this->readDataLine($handle, $title, $user_id);;
			} else { // first time 
				for($i = 1; $i < $linesNum; ++$i) {
					$lines[$i] = $this->readDataLine($handle, $title, $user_id);
				}
			}
			
			if(!$lines[1]) { // no current line
				$finished = true;
				break;
			}
			
			$firstDate = $lines[0] ? $lines[0]['date'] : $lines[1]['date'];
			$lastDate = 0;
			$lastValidLine = 0;
			for($i = $linesNum - 1; $i >= 1; --$i) {
				if(isset($lines[$i]['date'])) {
					$lastDate = $lines[$i]['date'];
					$lastValidLine = $i;
					break;
				}
			}
			
			/* LeePuman 注釋 Start */
			// 找出出错的lines
			// $errLines = array();
			// for($i = 1; $i <= $lastValidLine; ++$i) {
			// 	if($lines[$i]['date'] < $firstDate || $lines[$i]['date'] > $lastDate) { // error found
			// 		$errLines[] = $i;
			// 	}
			// }
			// if($firstDate > $lastDate) {
			// 	$fixed = false;
			// 	// 可能是最后一天日期错误, 并且是小于应有日期的错误
			// 	for($i = $lastValidLine - 1; $i >= 1; --$i) {
			// 		if($lines[$i]['date'] >= $firstDate) { 
			// 			$lines[$lastValidLine]['date'] = $lines[$i]['date'];
			// 			$lastDate = $lines[$i]['date'];
			// 			$fixed = true;
			// 			break;
			// 		}
			// 	}
			// 	// 整个数据第一行就出错了. 这种情况只会出现一次
			// 	if(!$fixed && !$lines[0]) {
			// 		for($i = 2; $i <= $lastValidLine; +$i) {
			// 			if($lines[$i]['date'] <= $lastDate) {
			// 				$lines[1]['date'] = $lines[$i]['date'];
			// 				$firstDate = $lines[$i]['date'];
			// 				$fixed = true;
			// 				break;
			// 			}
			// 		}
			// 	}
			// 	if(!$fixed)
			// 		throw_exception('数据错误. 查看的日期不足以检查出错误位置. 当前数据: ' . join(',', $lines[1]));
			// }
			
			// // 尝试修正出错的lines
			// if($lastDate == $firstDate) { // 第一天和最后一天日期相同. 目前还只发现这种情况.
			// 	foreach($errLines as $i) {
			// 		$lines[$i]['date'] = $firstDate;
			// 	}
			// } else { // 第一天和最后一天日期不同. 目前还没发现这种情况, 但是以防万一.
			// 	foreach($errLines as $i) {
			// 		if($lines[$i - 1] && $lines[$i]['keyword_id'] == $lines[$i - 1]['keyword_id']) { // 先往前看1天
			// 			$lines[$i]['date'] = $lines[$i - 1]['date'];
			// 		} elseif($lines[$i + 1] && $lines[$i]['keyword_id'] == $lines[$i + 1]['keyword_id'] // 不行的话, 往后看1天
			// 				&& !in_array($i + 1, $errLines)) {
			// 			$lines[$i]['date'] = $lines[$i + 1]['date'];
			// 		} else { // 都不行的话, 尽量取前一天. 这个时候可能不准确.
			// 			$lines[$i]['date'] = $lines[$i - 1] ? $lines[$i - 1]['date'] : $lastDate;
			// 		}
			// 	}
			// }
			/* Leepuman 注釋 END */
			
			// 保存数据. 设定key以便将 base 和 effect 的数据合并
			$currLine = $lines[1];
			// $key = $currLine['keyword_id'] . ':' . $currLine['source'] . ':' . $currLine['search_type'] . ':' . $currLine['date'];
			$datalist[] = $currLine;
				
			if($adgroup_ids !== null) {
				$adgroup_ids[$currLine['adgroup_id']] = $currLine['adgroup_id'];
			}
			
			for($i = 0; $i < $linesNum - 1; ++$i)
				$lines[$i] = $lines[$i + 1];
			$lines[$linesNum - 1] = array();
			
			if(!$lines[1]) { //全部处理完毕
				$finished = true;
				break;
			} else  if($lines[0]['date'] != $lines[1]['date']) { // next day
				break;
			}
		}
			
		return $datalist;
	}
	
	/** 
	 * 更新给定用户给定推广计划的给定source的所有关键词报表数据.
	 * 
	 * @param int $user_id
	 * @param int $campaign_id
	 * @param string $source '1,2', 'SUMMARY'中的某一个元素
	 * @param string $timeSlot DAY, 7DAY, 15DAY, 30DAY, 90DAY中的某一个
	 * @return bool 如果成功更新返回true, 如果没有必要更新, 返回false.
	 */
	public function syncCampaignKeywordReport($user_id, $campaign_id, $source, $timeSlot = '')
	{
		if(!$timeSlot) {
			// $timeSlot = '90DAY';
			$timeSlot = self::getCampKeywordTimeSlot($user_id, $campaign_id, $source);
        }
		if($timeSlot == '') {
			return false;
		}
		Log::record("time slot: $timeSlot", Log::DEBUG);
		// 提交任务
		$searchType = 'SEARCH,CAT,NOSEARCH'; //join(',', array_keys(RptBaseModel::$_SEARCH_TYPES));
		$taskIdBase = $this->submitKeywordRptAsyncTask('base', $campaign_id, $source, $searchType, $timeSlot);
		$taskIdEffect = $this->submitKeywordRptAsyncTask('effect', $campaign_id, $source, $searchType, $timeSlot);
		// 等待提交的任务完成. 根据日期长短设定等待时间(秒)
        $sleepSec = $timeSlot == '90DAY' ? 600 : ($timeSlot == '30DAY' ? 300 : 180);
        // $sleepSec = 60;
		// sleep($sleepSec);
		$taskStatusBase = null;
		$taskStatusEffect = null;
		do {
			sleep($sleepSec);
			$taskStatusBase = $this->getURLbyTaskId($taskIdBase);
			$taskStatusEffect = $this->getURLbyTaskId($taskIdEffect);
		} while ($taskStatusBase['status'] != 'done' || $taskStatusEffect['status'] != 'done');
		
		Log::record('read to download base file: ' . $taskStatusBase['download_url'], Log::DEBUG);
		Log::record('read to download effect file: ' . $taskStatusEffect['download_url'], Log::DEBUG);
		// 先将文件下载下来再读取
		$tmpFileDir = defined('SCRIPT_TMP_DIR') ? SCRIPT_TMP_DIR : TEMP_PATH;
		$tmpFileBase = $tmpFileDir . 'keyword_base_' . date('Ymd') . '_' . $campaign_id . '_' . $source . '.txt';
		$tmpFileEffect = $tmpFileDir . 'keyword_effect_' . date('Ymd') . '_' . $campaign_id . '_' . $source . '.txt';
		download_file($tmpFileBase, $taskStatusBase['download_url']);
		download_file($tmpFileEffect, $taskStatusEffect['download_url']);
		Log::record('Base file downloaded to : ' . $tmpFileBase, Log::DEBUG);
		Log::record('Effect file downloaded to : ' . $tmpFileEffect, Log::DEBUG);
		Log::save();
		
		// TODO 在mysql没配置成能保证大transaction不出错的情况下, 暂时不使用transaction.
		// $this->startTrans();
		try {
			$adgroup_ids = array();
			$this->saveFileData($user_id, $tmpFileBase, $adgroup_ids);
			$this->saveFileData($user_id, $tmpFileEffect);
			// 更新最后插入成功的时间
			self::saveCampKeywordTimeSlot($user_id, $source, $adgroup_ids);
			// $this->commit();
			return true;
		} catch (Exception $e) {
			// $this->rollback();
			return false;
		}
	}

	/**
	 * 读取报表文件并保存到数据库
	 * 
	 * @param int $user_id 用户ID
	 * @param string $tmpFile 报表文件的全路径
	 * @param string(int) [写入]如果给定, 将报表中涉及到的adgroup_ids写入该变量.
	 * @return int 数据库受影响的行数
	 */
	/* 
	private function saveFileData($user_id, $tmpFile, &$adgroup_ids = null) {
		$fh = fopen($tmpFile, 'r');
		if(!$fh) {
			throw_exception('打开文件错误: ' . $tmpFile);
		}
		$finished = false;
		$title = $this->readTitleLine($fh);
		$lines = array(array(), array(), array(), array(), array());
		$datalist = array();
		$batchNum = 500;
		
		$noNickTitle = array_diff($title, array('nick'));  // 不保存nick
		$noNickTitle[] = 'taobao_user_id'; // 增加user_id
		$keyFields = array('adgroup_id', 'keyword_id', 'date', 'source', 'search_type');
		$sqlOnDup = ' ON DUPLICATE KEY UPDATE  ';
		foreach($noNickTitle as $field) {
			if(!in_array($field, $keyFields)) // 不更新主键
				$sqlOnDup .= "$field=VALUES($field),";
		}
		$sqlOnDup = substr($sqlOnDup, 0, - 1); 
		$sqlInsert = 'INSERT INTO rpt_keyword'.$user_id.' (' . join(',', $noNickTitle) . ') VALUES ';
		
		$affectedRows = 0;		
		do {
			// 读取一天的数据
			$datalist = $this->readOneDayData($fh, $finished, $title, $user_id, $lines, $adgroup_ids); 
			$n = count($datalist);
			if(!$n)
				break;
			
			// 保存到DB
			$sql = '';
			for($i = 0; $i <= $n; ++$i) {
				if($i % $batchNum == 0 || $i == $n) { 
					// 加上sql尾巴, 并执行
					if($i / $batchNum >= 1 || $i == $n) { 
						$sql .= $sqlOnDup;
						$res = $this->execute($sql);
						if($res === false) {
							throw_exception('数据库错误. SQL: ' . $sql);
						} else {
							$affectedRows += $res;
						}
						if($i == $n) // #done
							break;
					}
					// 重置sql
					$sql = $sqlInsert;
				}
				
				$valStr = '';
				$first = true;
				foreach($noNickTitle as $field) {
					if(!$first) 
						$valStr .= ',';
					else
						$first = false;
					if($field == 'date' || $field == 'keywordstr') {
						$valStr .= '\'' . mysql_real_escape_string($datalist[$i][$field]) . '\'';
					}  else {
						$valStr .= $datalist[$i][$field] ? $datalist[$i][$field] : 0;
					}
				}
				$sql .= "($valStr)";
				if($i % $batchNum != ($batchNum - 1) && $i != ($n - 1)) {
					$sql .= ',';
				}
			}
		} while($datalist);
			
		fclose($fh);
		
		return $affectedRows;
		
	}
	*/

	private function saveFileData($user_id, $tmpFile, &$adgroup_ids = null)
	{
		$fh = fopen($tmpFile, 'r');
		if(!$fh) {
			throw_exception('打开文件错误: ' . $tmpFile);
		}
		$finished = false;
		$title = $this->readTitleLine($fh);
		$lines = array(array(), array(), array(), array(), array());
		$datalist = array();
		$batchNum = 500;
		$noNickTitle = array_diff($title, array('nick'));  // 不保存nick
		$noNickTitle[] = 'taobao_user_id'; // 增加user_id
		$keyFields = array('adgroup_id', 'keyword_id', 'date', 'source', 'search_type');
		$sqlOnDup = ' ON DUPLICATE KEY UPDATE  ';
		foreach($noNickTitle as $field) {
			if(!in_array($field, $keyFields)) // 不更新主键
				$sqlOnDup .= "$field=VALUES($field),";
		}
		$sqlOnDup = substr($sqlOnDup, 0, - 1); 
		$sqlInsert = 'INSERT INTO `rpt_keyword'.$user_id.'` (' . join(',', $noNickTitle) . ') VALUES ';
		$affectedRows = 0;

		$searchTypeVal = RptBaseModel::$_SEARCH_TYPES;
		$sourceTypeVal = RptBaseModel::$_SOURCE_TYPES;	
		while (!feof($fh)) {
	        $buffer = trim(fgets($fh));
	        if(strlen($buffer) > 0) {
	        	$data = explode(',', $buffer);
				$data = array_combine($title, $data);
				$data['source'] = $sourceTypeVal[$data['source']];
				$data['search_type'] = $searchTypeVal[$data['search_type']];
				$data['taobao_user_id'] = $user_id;
				unset($data['nick']);
				$datalist[] = $data;
				++$affectedRows;
				if(!isset($adgroup_ids[$data['adgroup_id']]))
					$adgroup_ids[$data['adgroup_id']] = $data['adgroup_id'];
	        }        	
	    }
		fclose($fh);
		if($affectedRows > 0) {
			$sql = '';
			$first = true;
			$valStr = '';
			$totalPage = $affectedRows > 10000 ? ceil($affectedRows/10000) : 1;
			for ($i=0; $i < $totalPage; $i++) {
				$start = $i * 10000;
				$tmp = array_slice($datalist, $start, 10000);
				// echo count($tmp)."<br>";
				
				$sql = '';
				foreach ($tmp as $key => $line) {
					$valStr = '';
					foreach($noNickTitle as $field) {
						// echo $field;
						
						if($field == 'date' || $field == 'keywordstr') {
							$valStr .= "'".mysql_escape_string($line[$field])."',";
						} else {
							$valStr .= ($line[$field] ? $line[$field] : 0) . ',';
						}
					}
					$valStr = substr($valStr, 0, - 1);
					$sql .= "($valStr),";
				}
				$sql = substr($sql, 0, - 1);
				$sql = $sqlInsert . $sql . $sqlOnDup;
				$this->query($sql);
			}
		}
			
		
		// $sql = substr($sql, 0, - 1); 
		// $sql = $sqlInsert . $sql . $sqlOnDup;
		return $affectedRows;
	}

	/**
	 * 提交一个关键词报表数据获取的异步API请求.
	 * 
	 * @param string $type 'base', 'effect'两者之一
	 * @param int $campaignId 
	 * @param string $source
	 * @param string $searchType
	 * @param string $timeSlot
	 * @throws TopClientException
	 * @return number 返回的异步任务ID
	 */
	private function submitKeywordRptAsyncTask($type, $campaignId, $source, $searchType, $timeSlot)
	{
		try {
			$req['nick'] = $this->me['nick'];
			$req['source'] = $source;
			$req['campaign_id'] = $campaignId;
			$req['search_type'] = $searchType;
			$req['time_slot'] = $timeSlot;
			$reqClass = $type == 'base' ? 'TopatsSimbaCampkeywordbaseGetRequest' : 'TopatsSimbaCampkeywordeffectGetRequest';
			$resp = $this->_api->$reqClass($req);
			Log::record('task_id : '.$resp['task']->task_id, Log::DEBUG);
			return $resp['task']->task_id;
		} catch (TopClientException $e) {
            $msg=$e->getMessage();
            Log::record($msg, Log::DEBUG);
			$msg=json_decode($msg);
			if($msg->error_response->sub_code=="isv.task-duplicate"){
                $n = strpos($msg->error_response->sub_msg,'=');
				return intval(substr($msg->error_response->sub_msg, $n+1));
            } else {
				throw $e;
			}
		}
	}
	
	/**
	 * get the task file by task ID
	 * @param int $taskId
	 */
	public function getURLbyTaskId($taskId)
	{
		$req['task_id'] = $taskId;
		$resp = $this->_api->TopatsResultGetRequest($req);
		return (array) $resp['task'];
	}
}
